/*
 *  This file is part of WWWPal.
 *
 *  Copyright (C) 2003  Dr. John Punin, Department of Computer Science
 *  Rensselaer Polytechnic Institute
 *
 *  WWWPal is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  WWWPal is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Foobar; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include "LOGR_logs.h"
#include "LOGR_webstat.h"
#include "LOGR_sites.h"
#include "LOGR_browsers.h"
#include "LOGR_simple_pages.h"
#include "LOGR_independent_pages.h"
#include "LOGR_independent_links.h"
#include "LOGR_gen_pages.h"
#include "LOGR_functions.h"
#include "LOGR_keywords.h"
#include "LOGR_gen_info.h"
#include "LOGR_gen_links.h"
#include "LOGR_users.h"
#include "LOGR_user_graph.h"
#include "LOGR_paths.h"

void
LOGR_add_keywords(WebStatPtr wsf, char *url, HTAssocList *al);

void
LOGR_add_user_entry_page_by_name(WebUserPtr usn,WebStatPtr wsf, char *url);

void
LOGR_extract_browsers_type_version_platform(WebStatPtr wsf,
					    WebBrowserNodePtr wbn);

char *
find_string_in_list(LNode_type *lstr, char *str);

WebGenInfoPtr
find_gen_code_in_list(LNode_type *lcodes,int code);

WebGenPagePtr
find_gen_page_in_list(LNode_type *lgen, char *url);
WebSiteNodePtr
find_site_in_list(LNode_type *lsites, char *site);
WebBrowserNodePtr
find_browser_in_list(LNode_type *lbrowsers, char *browser);
WebIndependentPagePtr
find_independent_page_in_list(LNode_type *lindependent_pages, char *url);

WebIndependentLinkPtr
find_independent_link_in_list(LNode_type *lindependent_links, char *urls, char *urlt);

WebIndependentLinkPtr
find_independent_link_in_table(Hashtable_type tindependent_links, char *urls, char *urlt);

WebUgraphPtr
LOGR_find_ugraph_in_graphs_list(LNode_type *lgraphs, WebUgraphPtr ugn);

WebStatPtr
new_web_stat(void)
{
  WebStatPtr wsf = (WebStatPtr)calloc(1, sizeof(WebStatStruct));
  if(!wsf) {
      fprintf(stderr,"Out of Memory new_web_stat\n");
      abort();
  }

  wsf->lcidr = init_linked_list();
  wsf->start_dates = init_linked_list();
  wsf->end_dates = init_linked_list();
  wsf->lsites = init_linked_list();
  init_hash_table(wsf->tsites);
  wsf->ldomains = init_linked_list();
  wsf->luas = init_linked_list();
  init_hash_table(wsf->tuas);
  wsf->lbrowsers = init_linked_list();
  wsf->lspiders = init_linked_list();
  wsf->lplatforms = init_linked_list();
  wsf->alcodes = init_linked_list();
  wsf->almethods = init_linked_list();
  wsf->alversions = init_linked_list();
  wsf->simple_pages = init_linked_list();
  wsf->independent_pages = init_linked_list();
  init_hash_table(wsf->tindependent_pages);
  wsf->simple_links = init_linked_list();
  wsf->independent_links = init_linked_list();
  init_hash_table(wsf->tindependent_links);
  wsf->entry_pages = init_linked_list();
  init_hash_table(wsf->tentry_pages);
  wsf->lsites_ref = init_linked_list();
  init_hash_table(wsf->tsites_ref);
  wsf->lurl_int_ref = init_linked_list();
  init_hash_table(wsf->turl_int_ref);
  wsf->lurl_ext_ref = init_linked_list();
  init_hash_table(wsf->turl_ext_ref);
  wsf->lkeywords = init_linked_list();
  init_hash_table(wsf->keywords);
  init_hash_table(wsf->labels);
  wsf->lusers = init_linked_list();
  wsf->lgraphs = init_linked_list();
  wsf->lpaths = init_linked_list();
  wsf->ldirectories = init_linked_list();
  init_hash_table(wsf->tdirectories);
  wsf->lcusers = init_current_users();
  return wsf;
}

void
delete_list_keywords(LNode_type *lkeywords)
{
  LNode_type *ptr = NULL;

  if(!lkeywords) return;

  for(ptr = lkeywords->back ; ptr != lkeywords ; ptr = ptr->back)
  {
    WebKeywordPtr kn = (WebKeywordPtr)ptr->info;
    delete_web_keyword_node(kn);
    kn = NULL;
  }
  Delete_all_list(lkeywords); 
}

void
delete_list_codes(LNode_type *lcodes)
{
  LNode_type *ptr = NULL;

  if(!lcodes) return;

  for(ptr = lcodes->back ; ptr != lcodes ; ptr = ptr->back)
  {
    WebGenInfoPtr usn = (WebGenInfoPtr)ptr->info;
    delete_gen_info_node(usn);
    usn = NULL;
  }
  Delete_all_list(lcodes); 
}

void
delete_list_users(LNode_type *lusers)
{
  LNode_type *ptr = NULL;

  if(!lusers) return;

  for(ptr = lusers->back ; ptr != lusers ; ptr = ptr->back)
  {
    WebUserPtr usn = (WebUserPtr)ptr->info;
    delete_web_user_node(usn);
    usn = NULL;
  }
  Delete_all_list(lusers); 
}


void
delete_list_graphs(LNode_type *lgraphs)
{
  LNode_type *ptr = NULL;

  if(!lgraphs) return;

  for(ptr = lgraphs->back ; ptr != lgraphs ; ptr = ptr->back)
  {
    void * GArray = (void *)ptr->info;      
    delete_graph_array(GArray);
    GArray = NULL;
  }
  Delete_all_list(lgraphs); 
}

void
delete_list_paths(LNode_type *lpaths)
{
  LNode_type *ptr = NULL;

  if(!lpaths) return;

  for(ptr = lpaths->back ; ptr != lpaths; ptr = ptr->back)
  {
    WebPathPtr upn = (WebPathPtr)ptr->info;
    delete_web_path_node(upn);
    upn = NULL;
  }
  Delete_all_list(lpaths); 
}

void
delete_list_directories(LNode_type *ldirectories)
{
  LNode_type *ptr = NULL;

  if(!ldirectories) return;

  for(ptr = ldirectories->back ; ptr != ldirectories; ptr = ptr->back)
  {
    WebGenPagePtr dn = (WebGenPagePtr)ptr->info;
    char *dname = (char *)LOGR_get_info_gen_page(dn);
    if(dname) {
	free(dname); dname = NULL;
    }
    delete_web_gen_page_node(dn); 
    dn = NULL;
  }
  Delete_all_list(ldirectories); 
}

void
delete_list_sites(LNode_type *lsites)
{
  LNode_type *ptr = NULL;

  if(!lsites) return;

  for(ptr = lsites->back ; ptr != lsites ; ptr = ptr->back)
  {
    WebSiteNodePtr wsn = (WebSiteNodePtr)ptr->info;
    delete_web_site_node(wsn); 
    wsn = NULL;
  }
  Delete_all_list(lsites); 
}

void
delete_list_domains(LNode_type *ldomains)
{
  LNode_type *ptr = NULL;

  if(!ldomains) return;

  for(ptr = ldomains->back ; ptr != ldomains ; ptr = ptr->back)
  {
    WebGenInfoPtr in = (WebGenInfoPtr)ptr->info;
    char *ds = (char *)LOGR_get_gen_info_context(in);
    if(ds) {
	free(ds); ds = NULL;
    }
    delete_gen_info_node(in);
    in = NULL;
  }
  Delete_all_list(ldomains); 
}

void
delete_list_uas(LNode_type *luas)
{
  LNode_type *ptr = NULL;

  if(!luas) return;

  for(ptr = luas->back ; ptr != luas ; ptr = ptr->back)
  {
    WebBrowserNodePtr wbn = (WebBrowserNodePtr)ptr->info;
    delete_web_browser_node(wbn); 
    wbn = NULL;
  }
  Delete_all_list(luas); 
}

void
delete_list_browsers(LNode_type *lbrowsers)
{
  LNode_type *ptr = NULL;

  if(!lbrowsers) return;

  for(ptr = lbrowsers->back ; ptr != lbrowsers ; ptr = ptr->back)
  {
    WebBrowserNodePtr wbn = (WebBrowserNodePtr)ptr->info;
    delete_web_browser_node(wbn); 
    wbn = NULL;
  }
  Delete_all_list(lbrowsers); 
}

void
delete_list_spiders(LNode_type *lspiders)
{
  LNode_type *ptr = NULL;

  if(!lspiders) return;

  for(ptr = lspiders->back ; ptr != lspiders ; ptr = ptr->back)
  {
    WebBrowserNodePtr wbn = (WebBrowserNodePtr)ptr->info;
    delete_web_browser_node(wbn); 
    wbn = NULL;
  }
  Delete_all_list(lspiders); 
}

void
delete_list_platforms(LNode_type *lplatforms)
{
  LNode_type *ptr = NULL;

  if(!lplatforms) return;

  for(ptr = lplatforms->back ; ptr != lplatforms ; ptr = ptr->back)
  {
    WebBrowserNodePtr wbn = (WebBrowserNodePtr)ptr->info;
    delete_web_browser_node(wbn); 
    wbn = NULL;
  }
  Delete_all_list(lplatforms); 
}

void
delete_list_simple_pages(LNode_type *lsp)
{
  LNode_type *ptr = NULL;

  if(!lsp) return;

  for(ptr = lsp->back ; ptr != lsp ; ptr = ptr->back)
  {
    WebSimplePagePtr spn = (WebSimplePagePtr)ptr->info;
    delete_web_simple_page_node(spn); 
    spn = NULL;
  }
  Delete_all_list(lsp); 
}

void
delete_list_simple_links(LNode_type *lsp)
{
  LNode_type *ptr = NULL;

  if(!lsp) return;

  for(ptr = lsp->back ; ptr != lsp ; ptr = ptr->back)
  {
    WebGenLinkPtr spn = (WebGenLinkPtr)ptr->info;
    delete_web_gen_link(spn); 
    spn = NULL;
  }
  Delete_all_list(lsp); 
}


void
delete_list_independent_pages(LNode_type *lop)
{
  LNode_type *ptr = NULL;

  if(!lop) return;

  for(ptr = lop->back ; ptr != lop ; ptr = ptr->back)
  {
    WebIndependentPagePtr opn = (WebIndependentPagePtr)ptr->info;
    delete_web_independent_page_node(opn); 
    opn = NULL;
  }
  Delete_all_list(lop); 
}

void
delete_list_independent_links(LNode_type *lol)
{
  LNode_type *ptr = NULL;

  if(!lol) return;

  for(ptr = lol->back ; ptr != lol ; ptr = ptr->back)
  {
    WebIndependentLinkPtr ol = (WebIndependentLinkPtr)ptr->info;
    delete_web_independent_link(ol); 
    ol= NULL;
  }
  Delete_all_list(lol); 
}

void
delete_list_entry_pages(LNode_type *lep)
{
  LNode_type *ptr = NULL;

  if(!lep) return;

  for(ptr = lep->back ; ptr != lep ; ptr = ptr->back)
  {
    WebGenPagePtr epn = (WebGenPagePtr)ptr->info;
    delete_web_gen_page_node(epn); 
    epn = NULL;
  }
  Delete_all_list(lep); 
}

void
delete_array_hours(void *hours[])
{
  int i = 0;

  if(!hours) return;

  for(i=0;i<24; i++)
  {
      WebGenInfoPtr wgi = (WebGenInfoPtr)hours[i];
      delete_gen_info_node(wgi);
      hours[i]=NULL;
  }
}

void
delete_array_days(void *days[])
{
  int i = 0;

  if(!days) return;

  for(i=0;i<31; i++)
  {
      WebGenInfoPtr wgi = (WebGenInfoPtr)days[i];
      delete_gen_info_node(wgi);
      days[i]=NULL;
  }
}

void
delete_array_months(void *months[])
{
  int i = 0;

  if(!months) return;

  for(i=0;i<12; i++)
  {
      WebGenInfoPtr wgi = (WebGenInfoPtr)months[i];
      delete_gen_info_node(wgi);
      months[i]=NULL;
  }
}

void
delete_web_stat(WebStatPtr wsf)
{
  if(wsf->base_webhost) {
	free(wsf->base_webhost); wsf->base_webhost = NULL;
    }
  if(wsf->local_domain) {
	free(wsf->local_domain); wsf->local_domain = NULL;
    }
  if(wsf->url_prefix) {
	free(wsf->url_prefix); wsf->url_prefix = NULL;
    }
  if(wsf->logs_dir) {
	free(wsf->logs_dir); wsf->logs_dir = NULL;
    }
  delete_array_hours(wsf->hours);
  delete_array_days(wsf->days);
  delete_array_months(wsf->months);
  delete_list_strings(wsf->lcidr); wsf->lcidr = NULL;
  Delete_all_list(wsf->start_dates); wsf->start_dates = NULL;
  Delete_all_list(wsf->end_dates); wsf->end_dates = NULL;
  delete_list_sites(wsf->lsites); wsf->lsites = NULL;
  Delete_hash_table(wsf->tsites);
  delete_list_domains(wsf->ldomains); wsf->ldomains = NULL;
  delete_list_uas(wsf->luas); wsf->luas = NULL;
  Delete_hash_table(wsf->tuas);
  delete_list_browsers(wsf->lbrowsers); wsf->lbrowsers = NULL;
  delete_list_spiders(wsf->lspiders); wsf->lspiders = NULL;
  delete_list_platforms(wsf->lplatforms); wsf->lplatforms = NULL;
  delete_list_simple_pages(wsf->simple_pages); wsf->simple_pages = NULL;
  delete_list_independent_pages(wsf->independent_pages); wsf->independent_pages = NULL;
  Delete_hash_table(wsf->tindependent_pages);
  delete_list_simple_links(wsf->simple_links); wsf->simple_links = NULL;
  delete_list_independent_links(wsf->independent_links); wsf->independent_links = NULL;
  Delete_hash_table(wsf->tindependent_links);
  delete_list_independent_pages(wsf->lurl_ext_ref); wsf->lurl_ext_ref = NULL;
  Delete_hash_table(wsf->turl_ext_ref);
  delete_list_entry_pages(wsf->lurl_int_ref); wsf->lurl_int_ref = NULL;
  Delete_hash_table(wsf->turl_int_ref);
  delete_list_entry_pages(wsf->entry_pages); wsf->entry_pages = NULL;
  Delete_hash_table(wsf->tentry_pages);
  delete_list_sites(wsf->lsites_ref); wsf->lsites_ref = NULL;
  Delete_hash_table(wsf->tsites_ref);
  delete_list_codes(wsf->alcodes); wsf->alcodes = NULL;
  delete_list_codes(wsf->almethods); wsf->almethods = NULL;
  delete_list_codes(wsf->alversions); wsf->alversions = NULL;
  delete_list_keywords(wsf->lkeywords); wsf->lkeywords = NULL;
  Delete_hash_table(wsf->keywords);
  Delete_hash_table(wsf->labels);
  delete_list_users(wsf->lusers); wsf->lusers = NULL;
  delete_list_graphs(wsf->lgraphs); wsf->lgraphs = NULL;
  delete_list_paths(wsf->lpaths); wsf->lpaths = NULL;
  delete_list_current_users(wsf->lcusers); wsf->lcusers = NULL;
  delete_list_directories(wsf->ldirectories); wsf->ldirectories = NULL;
  Delete_hash_table(wsf->tdirectories);
  free(wsf); wsf = NULL;
}


void
reset_web_stat(WebStatPtr wsf)
{
  wsf->count_hits = 0;
  wsf->size_bytes = 0;
  wsf->maxidnum = 0;
  wsf->resolve_timeout = 0;
  delete_array_hours(wsf->hours);
  delete_array_days(wsf->days);
  delete_array_months(wsf->months);
  delete_list_strings(wsf->lcidr); 
  wsf->lcidr =init_linked_list(); 
  Delete_all_list(wsf->start_dates);
  wsf->start_dates = init_linked_list();
  Delete_all_list(wsf->end_dates);
  wsf->end_dates =init_linked_list();
  delete_list_sites(wsf->lsites); 
  wsf->lsites = init_linked_list();
  Delete_hash_table(wsf->tsites);
  init_hash_table(wsf->tsites);
  delete_list_domains(wsf->ldomains); 
  wsf->ldomains = init_linked_list();
  delete_list_uas(wsf->luas); 
  wsf->luas = init_linked_list();
  Delete_hash_table(wsf->tuas);
  init_hash_table(wsf->tuas);
  delete_list_browsers(wsf->lbrowsers); 
  wsf->lbrowsers = init_linked_list();
  delete_list_spiders(wsf->lspiders); 
  wsf->lspiders = init_linked_list();
  delete_list_platforms(wsf->lplatforms); 
  wsf->lplatforms = init_linked_list();
  delete_list_simple_pages(wsf->simple_pages); 
  wsf->simple_pages = init_linked_list();
  delete_list_independent_pages(wsf->independent_pages); 
  wsf->independent_pages = init_linked_list();
  Delete_hash_table(wsf->tindependent_pages);
  init_hash_table(wsf->tindependent_pages);
  delete_list_simple_links(wsf->simple_links); 
  wsf->simple_links = init_linked_list();
  delete_list_independent_links(wsf->independent_links); 
  wsf->independent_links = init_linked_list();
  Delete_hash_table(wsf->tindependent_links);
  init_hash_table(wsf->tindependent_links);
  delete_list_independent_pages(wsf->lurl_ext_ref); 
  wsf->lurl_ext_ref = init_linked_list();
  Delete_hash_table(wsf->turl_ext_ref);
  init_hash_table(wsf->turl_ext_ref);
  delete_list_entry_pages(wsf->lurl_int_ref); 
  wsf->lurl_int_ref = init_linked_list();
  Delete_hash_table(wsf->turl_int_ref);
  init_hash_table(wsf->turl_int_ref);
  delete_list_entry_pages(wsf->entry_pages); 
  wsf->entry_pages = init_linked_list();
  Delete_hash_table(wsf->tentry_pages);
  init_hash_table(wsf->tentry_pages);
  delete_list_codes(wsf->alcodes); 
  wsf->alcodes = init_linked_list();
  delete_list_codes(wsf->almethods); 
  wsf->almethods = init_linked_list();
  delete_list_codes(wsf->alversions); 
  wsf->alversions = init_linked_list();
  delete_list_sites(wsf->lsites_ref); 
  wsf->lsites_ref = init_linked_list();
  Delete_hash_table(wsf->tsites_ref);
  init_hash_table(wsf->tsites_ref);
  delete_list_keywords(wsf->lkeywords); 
  wsf->lkeywords = init_linked_list();
  Delete_hash_table(wsf->keywords);
  init_hash_table(wsf->keywords);
  Delete_hash_table(wsf->labels);
  init_hash_table(wsf->labels);
  delete_list_users(wsf->lusers); 
  wsf->lusers = init_linked_list();
  delete_list_graphs(wsf->lgraphs); 
  wsf->lgraphs = init_linked_list();
  delete_list_paths(wsf->lpaths); 
  wsf->lpaths = init_linked_list();
  delete_list_current_users(wsf->lcusers); 
  wsf->lcusers = init_current_users();
  delete_list_directories(wsf->ldirectories); 
  wsf->ldirectories = init_linked_list();
  Delete_hash_table(wsf->tdirectories);
  init_hash_table(wsf->tdirectories);

}

void
clear_web_stat(WebStatPtr wsf)
{
  reset_web_stat(wsf);

  if(wsf->base_webhost) {
	free(wsf->base_webhost); wsf->base_webhost = NULL;
    }
  if(wsf->local_domain) {
	free(wsf->local_domain); wsf->local_domain = NULL;
    }
  if(wsf->url_prefix) {
	free(wsf->url_prefix); wsf->url_prefix = NULL;
    }
  if(wsf->logs_dir) {
	free(wsf->logs_dir); wsf->logs_dir = NULL;
    }
  wsf->logs_file_type = LOGR_EXTENDED_LOG;
}

void
LOGR_set_max_id_number(WebStatPtr wsf,int maxidnum)
{
  if(!wsf) return;
  wsf->maxidnum = maxidnum;
}

int
LOGR_get_max_id_number(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->maxidnum;
}

void
LOGR_set_resolve_timeout(WebStatPtr wsf,int resolve_timeout)
{
  if(!wsf) return;
  wsf->resolve_timeout = resolve_timeout;
}

int
LOGR_get_resolve_timeout(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->resolve_timeout;
}


void
LOGR_set_logs_directory(WebStatPtr wsf,char *ldir)
{
  if(!wsf) return;
  if(!ldir) return;
  GRStrAllocCopy(wsf->logs_dir,ldir);
}

const char *
LOGR_get_logs_directory(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->logs_dir;
}

void
LOGR_set_local_domain(WebStatPtr wsf,char *ldom)
{
  if(!wsf) return;
  if(!ldom) return;
  GRStrAllocCopy(wsf->local_domain,ldom);
}

const char *
LOGR_get_local_domain(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->local_domain;
}

void
LOGR_set_base_webhost(WebStatPtr wsf,char *bwh)
{
  if(!wsf) return;
  if(!bwh) return;
  GRStrAllocCopy(wsf->base_webhost,bwh);
}

const char *
LOGR_get_base_webhost(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->base_webhost;
}

const char *
LOGR_get_url_prefix(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->url_prefix;
}

void
LOGR_set_url_prefix(WebStatPtr wsf,char *ulp)
{
  if(!wsf) return;
  if(!ulp) return;
  GRStrAllocCopy(wsf->url_prefix,ulp);
}


void
LOGR_set_logs_filetype(WebStatPtr wsf,int lft)
{
  if(!wsf) return;
  wsf->logs_file_type = lft;
}

int
LOGR_get_logs_filetype(WebStatPtr wsf)
{
  if(!wsf) return -1;
  return (wsf->logs_file_type);
}


void
LOGR_set_compute_user_sessions(WebStatPtr wsf)
{
  if(!wsf) return;
  wsf->compute_user_sessions = 1;
}

void
LOGR_reset_compute_user_sessions(WebStatPtr wsf)
{
  if(!wsf) return;
  wsf->compute_user_sessions = 0;
}

short
LOGR_get_compute_user_sessions(WebStatPtr wsf)
{
  if(!wsf) return 0;
  return (wsf->compute_user_sessions);
}

void
LOGR_set_resolve_host_ip(WebStatPtr wsf)
{
  if(!wsf) return;
  wsf->resolve_host_ip = 1;
}

void
LOGR_reset_resolve_host_ip(WebStatPtr wsf)
{
  if(!wsf) return;
  wsf->resolve_host_ip = 0;
}

short
LOGR_get_resolve_host_ip(WebStatPtr wsf)
{
  if(!wsf) return 0;
  return (wsf->resolve_host_ip);
}

void
LOGR_set_start_date(WebStatPtr wsf,time_t t)
{
  if(!wsf) return;
  Insert_linked_list(wsf->start_dates,create_new_node_llist((char *)t));
}


LNode_type *
LOGR_get_start_dates(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->start_dates;
}

time_t
LOGR_get_first_start_date(WebStatPtr wsf)
{
    LNode_type *lsdates = NULL;
    if(!wsf) return 0;
    lsdates = wsf->start_dates;
    if(lsdates->next != lsdates)
	return (time_t)lsdates->next->info;
    return 0;
}

time_t
LOGR_get_minimum_start_date(WebStatPtr wsf)
{
    LNode_type *lsdates = NULL;
    LNode_type *ptr = NULL;
    time_t ct, mt;
    if(!wsf) return 0;
    mt= LOGR_get_first_start_date(wsf);
    lsdates = wsf->start_dates;

    for(ptr = lsdates->back ; ptr != lsdates; ptr = ptr->back)
    {
	ct = (time_t)ptr->info;
	if(ct < mt)
	    mt = ct;
    }
    return mt;
}

time_t
LOGR_get_first_end_date(WebStatPtr wsf)
{
    LNode_type *ledates = NULL;
    if(!wsf) return 0;
    ledates = wsf->end_dates;
    if(ledates->next != ledates)
	return (time_t)ledates->next->info;
    return 0;
}

time_t
LOGR_get_maximum_end_date(WebStatPtr wsf)
{
    LNode_type *ledates = NULL;
    LNode_type *ptr = NULL;
    time_t ct, mt;
    if(!wsf) return 0;
    mt = LOGR_get_first_end_date(wsf);
    ledates = wsf->end_dates;

    for(ptr = ledates->back ; ptr != ledates; ptr = ptr->back)
    {
	ct = (time_t)ptr->info;
	if(ct > mt)
	    mt = ct;
    }
    return mt;
}

void
LOGR_set_end_date(WebStatPtr wsf,time_t t)
{
  if(!wsf) return;
  Insert_linked_list(wsf->end_dates,create_new_node_llist((char *)t));
}

LNode_type *
LOGR_get_end_dates(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->end_dates;
}

char *
find_string_in_list(LNode_type *lstr, char *str)
{
  LNode_type *ptr = NULL;

  if(!str || !lstr) return NULL;

  for(ptr = lstr->back; ptr != lstr; ptr = ptr->back)
  {
      char *cstr = (char *)ptr->info;
      if(cstr && !strcmp(str,cstr))
	  return cstr;
  }
  return NULL;
}

void
LOGR_clear_list_cidr(WebStatPtr wsf)
{
    delete_list_strings(wsf->lcidr); 
    wsf->lcidr =init_linked_list(); 
}

void
LOGR_add_cidr(WebStatPtr wsf, char *cidr)
{
    char *scidr = NULL;

    if(!wsf || !cidr)
	return;

    scidr = find_string_in_list(wsf->lcidr,cidr);
    if(!scidr)
    {
	GRStrAllocCopy(scidr,cidr);
	Insert_linked_list(wsf->lcidr,create_new_node_llist((char *)scidr));
    }
}

short
LOGR_is_ip_local(WebStatPtr wsf, char *ip)
{
    LNode_type *ptr = NULL;
    short isa = 0;
    char *p = NULL;
    int plen = 0;

    if(!wsf || !ip)
	return 0;

    /*printf("testing IP %s\n", ip);*/

    for(ptr = wsf->lcidr->back ; ptr!= wsf->lcidr; ptr = ptr->back)
    {
	char *scidr = (char *)ptr->info;
	p = strchr(scidr,'/');
	if(p)
	    *p = '\0';
	plen = atoi(p+1);
	isa = LOGR_is_address_in_subnet(ip,scidr,plen);
	*p = '/';
	if(isa)
	{
	    /*printf("IP %s is local\n", ip);*/
	    return isa;
	}
    }
    return 0;
}

WebGenInfoPtr
find_domain_in_list(LNode_type *ldomains, char *domain)
{
  LNode_type *ptr = NULL;

  if(!domain || !ldomains) return NULL;

  for(ptr = ldomains->back; ptr != ldomains; ptr = ptr->back)
  {
      WebGenInfoPtr in =  (WebGenInfoPtr)ptr->info;
      char *ds = (char *)LOGR_get_gen_info_context(in);
      if(ds && !strcasecmp(domain,ds))
	  return in;
  }
  return NULL;
}

void
LOGR_add_domain(WebStatPtr wsf, char *domain,long nacc,long size,long npages)
{
    WebGenInfoPtr in = NULL;
    if(!wsf || !domain)
	return;
    in = find_domain_in_list(wsf->ldomains,domain);
    if(!in)
    {
	char *ldomain = NULL;
	GRStrAllocCopy(ldomain,domain);
	LOGR_to_lower(ldomain);
	/* create new node site */
	/*printf("New Domain %s\n",domain);*/
	in = new_gen_info_node();
	LOGR_set_gen_info_context(in,ldomain);
	Insert_linked_list(wsf->ldomains,create_new_node_llist((char *)in));
    }
    LOGR_add_gen_info_sizebytes(in,size);
    LOGR_add_gen_info_naccesses(in,nacc);
    LOGR_add_gen_info_npages(in,npages);
}

WebSiteNodePtr
find_site_in_list(LNode_type *lsites, char *site)
{
  LNode_type *ptr = NULL;

  if(!site || !lsites) return NULL;

  for(ptr = lsites->back; ptr != lsites; ptr = ptr->back)
  {
      WebSiteNodePtr cwsn =  (WebSiteNodePtr)ptr->info;

      const char *csite = cwsn ? LOGR_get_site_dns(cwsn) : NULL;
      const char *ipsite = cwsn ? LOGR_get_site_ip(cwsn) : NULL;

      if(csite && !strcmp(site, csite))
      {
	  return cwsn;
      }
      if(ipsite && !strcmp(site, ipsite))
      {
	  return cwsn;
      }
  }
  return NULL;
}

void
LOGR_add_site(WebStatPtr wsf, char *site,int cur_size,int ishtmlpage)
{
    WebSiteNodePtr wsn = NULL;
/*    if(wsf)
      wsn = find_site_in_list(wsf->lsites,site);*/

    if(wsf)
	wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites,site);

    if(!wsn)
    {
	/* create new node site */
	wsn = new_web_site_node();
	/*printf("New site %s\n", site);*/

	if(LOGR_is_ip(site))
	    LOGR_set_site_ip(wsn, site);
	else
	    LOGR_set_site_dns(wsn, site);

	Insert_linked_list(wsf->lsites,create_new_node_llist((char *)wsn));
	insert_hash_info(wsf->tsites, site, (char *)wsn);
    }

    LOGR_inc_site_naccesses(wsn);
    LOGR_add_site_sizebytes(wsn,cur_size);
    if(ishtmlpage)
	LOGR_inc_site_nacc_search(wsn);
}

void
LOGR_add_size_site(WebStatPtr wsf, char *site, char *size)
{
  WebSiteNodePtr wsn = NULL;
  long csize =0;
  long nsize = 0;

  if(!wsf || !site || !size)
      return;
/*  wsn = find_site_in_list(wsf->lsites,site); */

  wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites,site);

  if(!wsn)
  {
      fprintf(stderr,"Site %s is missing\n", site);
      return;
  }
  csize = LOGR_get_site_sizebytes(wsn);
  nsize = atoi(size);
  LOGR_set_site_sizebytes(wsn,csize+nsize);
}

void
LOGR_add_num_size_site(WebStatPtr wsf, char *site, int size)
{
  WebSiteNodePtr wsn = NULL;
  long csize =0;

  if(!wsf || !site)
      return;
/*  wsn = find_site_in_list(wsf->lsites,site);*/

  wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites,site);
  if(!wsn)
  {
      fprintf(stderr,"Site %s is missing\n", site);
      return;
  }
  csize = LOGR_get_site_sizebytes(wsn);
  LOGR_set_site_sizebytes(wsn,csize+size);
}


LNode_type *
LOGR_get_list_sites(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lsites;
}

LNode_type *
LOGR_get_list_domains(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->ldomains;
}

void
LOGR_add_site_ref(WebStatPtr wsf, char *site)
{
    WebSiteNodePtr wsn = NULL;
    /*if(wsf)
      wsn = find_site_in_list(wsf->lsites_ref,site);*/
    if(wsf)
	wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites_ref, site);

    if(!wsn)
    {
	/* create new node site */
	wsn = new_web_site_node();
	/*printf("New site %s\n", site);*/
	LOGR_set_site_dns(wsn, site);
	Insert_linked_list(wsf->lsites_ref,create_new_node_llist((char *)wsn));
	insert_hash_info(wsf->tsites_ref, site, (char *)wsn);
    }
    LOGR_inc_site_naccesses(wsn);
}

void
LOGR_add_site_ref2(WebStatPtr wsf, char *site,int size)
{
    WebSiteNodePtr wsn = NULL;
    /*if(wsf)
      wsn = find_site_in_list(wsf->lsites_ref,site);*/

    if(wsf)
	wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites_ref, site);

    if(!wsn)
    {
	/* create new node site */
	wsn = new_web_site_node();
	/*printf("New site <%s>\n", site);*/
	LOGR_set_site_dns(wsn, site);
	Insert_linked_list(wsf->lsites_ref,create_new_node_llist((char *)wsn));
	insert_hash_info(wsf->tsites_ref, site, (char *)wsn);
    }
    LOGR_inc_site_naccesses(wsn);
    {
      int csize = LOGR_get_site_sizebytes(wsn);
      LOGR_set_site_sizebytes(wsn,csize+size);
    }
}


void
LOGR_add_site_ref3(WebStatPtr wsf, char *site,int size,TypeReferer tr)
{
    WebSiteNodePtr wsn = NULL;

    char *p = site ? strchr(site,'?') : NULL;
    if(p) *p='\0';

    /*if(wsf)
      wsn = find_site_in_list(wsf->lsites_ref,site);*/
    if(wsf)
	wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites_ref, site);
    if(!wsn)
    {
	/* create new node site */
	wsn = new_web_site_node();
	/*printf("New site <%s> TR %d\n", site, tr);*/
	LOGR_set_site_dns(wsn, site);
	Insert_linked_list(wsf->lsites_ref,create_new_node_llist((char *)wsn));
	insert_hash_info(wsf->tsites_ref, site, (char *)wsn);
    }
    LOGR_inc_site_naccesses(wsn);
    {
      int csize = LOGR_get_site_sizebytes(wsn);
      LOGR_set_site_sizebytes(wsn,csize+size);
      LOGR_set_site_type_referer2(wsn,tr);      
    }
    if(tr == RDYNAMIC_REF)
    {
      int csize = LOGR_get_site_sizebytes_search(wsn);
      LOGR_set_site_sizebytes_search(wsn,csize+size);
      LOGR_inc_site_nacc_search(wsn);
    }
}

void
LOGR_add_size_site_ref(WebStatPtr wsf, char *site, char *size)
{
  WebSiteNodePtr wsn = NULL;
  long csize =0;
  long nsize = 0;

  if(!wsf || !site || !size)
      return;
  /*wsn = find_site_in_list(wsf->lsites_ref,site);*/
  wsn = (WebSiteNodePtr)get_element_hashed(wsf->tsites_ref, site);
  if(!wsn)
  {
      fprintf(stderr,"Site %s is missing\n", site);
      return;
  }
  csize = LOGR_get_site_sizebytes(wsn);
  nsize = atoi(size);
  LOGR_set_site_sizebytes(wsn,csize+nsize);
}

LNode_type *
LOGR_get_list_sites_ref(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lsites_ref;
}

WebBrowserNodePtr
find_browser_in_list(LNode_type *lbrowsers, char *browser)
{
  LNode_type *ptr = NULL;

  if(!browser || !lbrowsers) return NULL;

  for(ptr = lbrowsers->back; ptr != lbrowsers; ptr = ptr->back)
  {
      WebBrowserNodePtr cwbn =  (WebBrowserNodePtr)ptr->info;
      const char *cua = cwbn ? LOGR_get_user_agent(cwbn) : NULL;
      if(cua && !strcmp(browser, cua))
      {
	  return cwbn;
      }
  }
  return NULL;
}

void
LOGR_add_browser(WebStatPtr wsf, char *browser, int nsize,int ishtmlpage)
{
    WebBrowserNodePtr wbn = NULL;

    /* find_browser_in_list(wsf->luas,browser); */
    if(wsf)
      	wbn = (WebBrowserNodePtr)get_element_hashed(wsf->tuas,browser);

    if(!wbn)
    {
	/* create new node site */
	wbn = new_web_browser_node();
        /*printf("New browser <%s>\n", browser);*/
	LOGR_set_user_agent(wbn, browser);
	LOGR_extract_browsers_type_version_platform(wsf,wbn);
	Insert_linked_list(wsf->luas,create_new_node_llist((char *)wbn));
	insert_hash_info(wsf->tuas, browser, (char *)wbn);
    }
    LOGR_inc_browser_naccesses(wbn);
    if(nsize > 0)
	LOGR_add_browser_sizebytes(wbn,(double)nsize);
    if(ishtmlpage)
	LOGR_inc_browser_npages(wbn);
}

void
LOGR_add_size_browser(WebStatPtr wsf, char *browser, char *size)
{
  WebBrowserNodePtr wbn = NULL;
  long csize =0;
  long nsize = 0;

  if(!wsf || !browser || !size)
      return;

  /*wbn = find_browser_in_list(wsf->luas,browser);*/
  wbn = (WebBrowserNodePtr)get_element_hashed(wsf->tuas,browser);

  if(!wbn)
  {
      fprintf(stderr,"Browser %s is missing\n", browser);
      return;
  }
  csize = LOGR_get_browser_sizebytes(wbn);
  nsize = atoi(size);
  LOGR_set_browser_sizebytes(wbn,(double)(csize+nsize));
}


void
LOGR_add_size_browser_num(WebStatPtr wsf, char *browser, int nsize)
{
  WebBrowserNodePtr wbn = NULL;
  long csize =0;

  if(!wsf || !browser)
      return;
  /*wbn = find_browser_in_list(wsf->luas,browser);*/

  wbn = (WebBrowserNodePtr)get_element_hashed(wsf->tuas,browser);

  if(!wbn)
  {
      fprintf(stderr,"Browser %s is missing\n", browser);
      return;
  }
  csize = LOGR_get_browser_sizebytes(wbn);
  LOGR_set_browser_sizebytes(wbn,(double)(csize+nsize));
}

LNode_type *
LOGR_get_list_ua(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->luas;
}

LNode_type *
LOGR_get_list_browser(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lbrowsers;
}

LNode_type *
LOGR_get_list_platform(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lplatforms;
}

LNode_type *
LOGR_get_list_spider(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lspiders;
}

double
LOGR_get_total_browser_size(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;
  double total = 0;
  if(!wsf) return 0;

  for(ptr = wsf->luas->back; ptr != wsf->luas; ptr = ptr->back)
  {
      WebBrowserNodePtr cwbn =  (WebBrowserNodePtr)ptr->info;
      total+=LOGR_get_browser_sizebytes(cwbn);
  }
  return total;
}

long
LOGR_get_total_browser_naccesses(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;
  long total = 0;
  if(!wsf) return 0;

  for(ptr = wsf->luas->back; ptr != wsf->luas; ptr = ptr->back)
  {
      WebBrowserNodePtr cwbn =  (WebBrowserNodePtr)ptr->info;
      total+=LOGR_get_browser_naccesses(cwbn);
  }
  return total;
}

void
LOGR_add_gen_browser(WebStatPtr wsf,WebBrowserNodePtr wbn)
{
  char browser_name[100];
  WebBrowserNodePtr gwbn = NULL;

  if(!wsf || !wbn)
      return;

  sprintf(browser_name, "%s %s",
	  LOGR_string_browser(wbn), LOGR_string_version(wbn));
  gwbn = find_browser_in_list(wsf->lbrowsers,browser_name);
  if(!gwbn)
  {
      gwbn = new_web_browser_node();
      /*printf("New General browser <%s>\n", browser_name);*/
      LOGR_set_user_agent(gwbn, browser_name);
      Insert_linked_list(wsf->lbrowsers,create_new_node_llist((char *)gwbn));
  }
  {
      long na = LOGR_get_browser_naccesses(wbn);
      long nga = LOGR_get_browser_naccesses(gwbn);
      long npages = LOGR_get_browser_npages(wbn);
      long ngpages = LOGR_get_browser_npages(gwbn);
      double size = LOGR_get_browser_sizebytes(wbn);
      double nsize = LOGR_get_browser_sizebytes(gwbn);
      LOGR_set_browser_naccesses(gwbn, na+nga);
      LOGR_set_browser_sizebytes(gwbn,size+nsize);
      LOGR_set_browser_npages(gwbn,npages+ngpages);
  }
}

void
LOGR_add_gen_spider(WebStatPtr wsf,WebBrowserNodePtr wbn)
{
  WebBrowserNodePtr gwbn = NULL;
  char *name_spider = NULL;

  if(!wsf || !wbn)
      return;

  name_spider = (char *)LOGR_get_user_agent(wbn);

  gwbn = find_browser_in_list(wsf->lspiders,name_spider);
  if(!gwbn)
  {
      gwbn = new_web_browser_node();
      /*printf("New General spider <%s>\n", name_spider);*/
      LOGR_set_user_agent(gwbn, name_spider);
      Insert_linked_list(wsf->lspiders,create_new_node_llist((char *)gwbn));
  }
  {
      long na = LOGR_get_browser_naccesses(wbn);
      long nga = LOGR_get_browser_naccesses(gwbn);
      long npages = LOGR_get_browser_npages(wbn);
      long ngpages = LOGR_get_browser_npages(gwbn);
      double size = LOGR_get_browser_sizebytes(wbn);
      double nsize = LOGR_get_browser_sizebytes(gwbn);
      LOGR_set_browser_naccesses(gwbn, na+nga);
      LOGR_set_browser_sizebytes(gwbn,size+nsize);
      LOGR_set_browser_npages(gwbn,npages+ngpages);
  }
}

void
LOGR_add_gen_platform(WebStatPtr wsf,WebBrowserNodePtr wbn)
{
  WebBrowserNodePtr gwbn = NULL;
  char *nplat = NULL;
  if(!wsf || !wbn)
      return;

  nplat = (char *)LOGR_string_platform(wbn);
  gwbn = find_browser_in_list(wsf->lplatforms,nplat);
  if(!gwbn)
  {
      gwbn = new_web_browser_node();
      /*printf("New General Platform <%s>\n",nplat);*/
      LOGR_set_user_agent(gwbn, nplat);
      Insert_linked_list(wsf->lplatforms,create_new_node_llist((char *)gwbn));
  }
  {
      long na = LOGR_get_browser_naccesses(wbn);
      long nga = LOGR_get_browser_naccesses(gwbn);
      long npages = LOGR_get_browser_npages(wbn);
      long ngpages = LOGR_get_browser_npages(gwbn);
      double size = LOGR_get_browser_sizebytes(wbn);
      double nsize = LOGR_get_browser_sizebytes(gwbn);
      LOGR_set_browser_naccesses(gwbn, na+nga);
      LOGR_set_browser_sizebytes(gwbn,size+nsize);
      LOGR_set_browser_npages(gwbn,npages+ngpages);
  }
}

void
LOGR_extract_browsers_type_version_platform(WebStatPtr wsf,
					    WebBrowserNodePtr cwbn)
{
    if(!wsf || !cwbn)
	return;
    LOGR_extract_ua_browser(cwbn);
    LOGR_extract_ua_platform(cwbn);
    LOGR_extract_ua_version(cwbn);
    /*printf("User Agent: %s\n, Browser: %s, Version %s, Platform %s\n\n",
      cwbn->ua,LOGR_string_browser(cwbn), LOGR_string_version(cwbn),
      LOGR_string_platform(cwbn));*/
}

void
clear_browsers_platforms_spiders(WebStatPtr wsf)
{
    LNode_type *ptr = NULL;
    LNode_type *lbrowsers = wsf->lbrowsers;
    LNode_type *lspiders = wsf->lspiders;
    LNode_type *lplatforms = wsf->lplatforms;
    WebBrowserNodePtr wbn = NULL;

    for(ptr = lbrowsers->back ; ptr != lbrowsers; ptr = ptr->back)
    {
	wbn = (WebBrowserNodePtr)ptr->info;
	LOGR_set_browser_naccesses(wbn,0);
	LOGR_set_browser_npages(wbn,0);
	LOGR_set_browser_sizebytes(wbn,0);
    }

    for(ptr = lplatforms->back ; ptr != lplatforms; ptr = ptr->back)
    {
	wbn = (WebBrowserNodePtr)ptr->info;
	LOGR_set_browser_naccesses(wbn,0);
	LOGR_set_browser_npages(wbn,0);
	LOGR_set_browser_sizebytes(wbn,0);
    }

    for(ptr = lspiders->back ; ptr != lspiders; ptr = ptr->back)
    {
	wbn = (WebBrowserNodePtr)ptr->info;
	LOGR_set_browser_naccesses(wbn,0);
	LOGR_set_browser_npages(wbn,0);
	LOGR_set_browser_sizebytes(wbn,0);
    }
}

void
LOGR_process_browsers_type_version_platform(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;

  if(!wsf) 
      return;

  clear_browsers_platforms_spiders(wsf);

  for(ptr = wsf->luas->back; ptr != wsf->luas; ptr = ptr->back)
  {
      WebBrowserNodePtr cwbn =  (WebBrowserNodePtr)ptr->info;
      /*LOGR_extract_browsers_type_version_platform(wsf,cwbn);*/
      LOGR_add_gen_browser(wsf,cwbn);
      LOGR_add_gen_platform(wsf,cwbn);
      if(LOGR_is_spider(cwbn))
	  LOGR_add_gen_spider(wsf,cwbn);

  }
}

WebIndependentPagePtr
find_independent_page_in_list(LNode_type *lindependent_pages, char *url)
{
  LNode_type *ptr = NULL;

  if(!url || !lindependent_pages) return NULL;

  for(ptr = lindependent_pages->back; ptr != lindependent_pages; 
      ptr = ptr->back)
  {
      WebIndependentPagePtr copn =  (WebIndependentPagePtr)ptr->info;
      const char *curl = copn ? LOGR_get_ip_url(copn) : NULL;
      if(curl && !strcmp(url, curl))
      {
	  return copn;
      }
  }
  return NULL;
}

void
LOGR_add_independent_page(WebStatPtr wsf, char *url,long size, int http_code,
		     char *mime)
{
    WebIndependentPagePtr opn = NULL;

    /*opn = find_independent_page_in_list(wsf->independent_pages,url);*/
    if(wsf)
      	opn = (WebIndependentPagePtr)
	    get_element_hashed(wsf->tindependent_pages,url);    

    if(!opn)
    {
	/* create new independent page node  */
	opn = new_web_independent_page_node();
	/*printf("New independent_page %s\n", url);*/
	wsf->maxidnum++;
	LOGR_set_ip_num(opn,wsf->maxidnum);
	LOGR_set_ip_url(opn, url);
	LOGR_set_ip_sizebytes(opn,size);
	LOGR_set_ip_http_code(opn,http_code);
	LOGR_set_ip_mime(opn,mime);
	Insert_linked_list(wsf->independent_pages,
			   create_new_node_llist((char *)opn));
	insert_hash_info(wsf->tindependent_pages, url, (char *)opn);
    }
    else
    {
	int nacc = LOGR_get_ip_naccesses(opn);
	if(nacc == 0)
	{
	    /*printf("Updating independent page %s\n", url);*/
	  LOGR_set_ip_sizebytes(opn,size);
	  LOGR_set_ip_http_code(opn,http_code);
	  LOGR_set_ip_mime(opn,mime);
	}
    }
    /* increase count on independent page */
    LOGR_inc_ip_naccesses(opn);
}

LNode_type *
LOGR_get_list_simple_pages(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->simple_pages;
}

LNode_type *
LOGR_get_list_simple_links(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->simple_links;
}


LNode_type *
LOGR_get_list_independent_pages(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->independent_pages;
}

void
LOGR_add_url_external_referer(WebStatPtr wsf, char *url,char *turl,int csize,
			      int http_code)
{
    WebIndependentPagePtr opn = NULL;

    /*WebIndependentPagePtr opn = find_independent_page_in_list(wsf->lurl_ext_ref,url);*/
    if(wsf)
	opn = (WebIndependentPagePtr)get_element_hashed(wsf->turl_ext_ref,url);

    if(!opn)
    {
	/*WebGenPagePtr gpn = find_gen_page_in_list(wsf->entry_pages,turl);*/
	WebGenPagePtr gpn = turl ? (WebGenPagePtr)get_element_hashed(wsf->tentry_pages, turl) : NULL;
	/* create new independent page node  */
	opn = new_web_independent_page_node();
	/*printf("New referer ext url %s\n", url);*/
	wsf->maxidnum++;	
	LOGR_set_ip_num(opn,wsf->maxidnum);
	LOGR_set_ip_url(opn, url);
	/* save referee url entry page */
	LOGR_set_ip_info(opn,(void *)gpn);
	LOGR_set_ip_http_code(opn,http_code);
	Insert_linked_list(wsf->lurl_ext_ref,
			   create_new_node_llist((char *)opn));
	insert_hash_info(wsf->turl_ext_ref,url, (char *)opn);
    }
    /* increase count on site */
    LOGR_inc_ip_naccesses(opn);
    LOGR_add_ip_sizebytes(opn, csize);
	
}

LNode_type *
LOGR_get_list_url_external_referers(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lurl_ext_ref;
}

void
LOGR_add_url_internal_referer(WebStatPtr wsf, char *url, GraphFrame *gf, 
			      int csize)
{
    WebGenPagePtr gpn =	NULL;

    /*WebGenPagePtr gpn = find_gen_page_in_list(wsf->lurl_int_ref,url);*/

    if(wsf)
	gpn = (WebGenPagePtr)get_element_hashed(wsf->turl_int_ref,url);

    if(!gpn)
    {
      TypePage tp = 0;
      void *iptr = LOGR_find_info_gen_page(url,&tp,wsf,gf);
      /* create new internal url node  */
      gpn = new_web_gen_page_node();
      /*printf("New URL INT <%s> type %d\n", url,tp); */
      if(iptr)
      {
	LOGR_set_type_gen_page(gpn,tp);
	LOGR_set_info_gen_page(gpn,iptr);
      }
      Insert_linked_list(wsf->lurl_int_ref,
			 create_new_node_llist((char *)gpn));
      insert_hash_info(wsf->turl_int_ref,url, (char *)gpn);
    }
    /* increase count on site */
    LOGR_inc_gen_naccesses(gpn);
    LOGR_add_gen_total_bytes(gpn,csize);
}

LNode_type *
LOGR_get_list_url_internal_referers(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lurl_int_ref;
}

WebGenPagePtr
find_gen_page_in_list(LNode_type *lgen, char *url)
{
  LNode_type *ptr = NULL;

  if(!url || !lgen) return NULL;

  for(ptr = lgen->back; ptr != lgen; ptr = ptr->back)
  {
      WebGenPagePtr cgpn =  (WebGenPagePtr)ptr->info;
      const char *curl = cgpn ? LOGR_get_gen_url(cgpn) : NULL;
      if(curl && !strcmp(url, curl))
      {
	  return cgpn;
      }
  }
  return NULL;
}

void *
LOGR_find_info_gen_page(char *url,TypePage *tp,WebStatPtr wsf,GraphFrame *gf)
{
    PNODE v = NULL;
    WebIndependentPagePtr opn = NULL;

    if(!url || !wsf || !gf)
	return NULL;

    v = find_vertex_by_name(gf,url);
    if(v)
    {
	*tp = PWEBNODE;
	return (void *) v;
    }
    /*opn = find_independent_page_in_list(wsf->independent_pages,url);*/
    opn = (WebIndependentPagePtr)
	get_element_hashed(wsf->tindependent_pages,url);    
    if(opn)
    {
	*tp = PWEBINDEPENDENT;
	return (void *) opn;
    }
    else
    {
	char *mime = HTGetMime(url);	
	*tp = PWEBINDEPENDENT;
      	/* create new independent page node  */
	opn = new_web_independent_page_node();
	wsf->maxidnum++;
	LOGR_set_ip_num(opn,wsf->maxidnum);
	/*printf("NEW INDEPENDENT page %s\n", url);*/
	LOGR_set_ip_url(opn, url);
	LOGR_set_ip_mime(opn,mime);
	Insert_linked_list(wsf->independent_pages,
			   create_new_node_llist((char *)opn));
	insert_hash_info(wsf->tindependent_pages, url, (char *)opn);
	return (void *)opn;
    }
    return NULL;
}

void *
LOGR_find_info_gen_link(char *urls,char *urlt,TypeLink *tl,WebStatPtr wsf,
			GraphFrame *gf)
{
    PEDGE e = NULL;
    WebIndependentLinkPtr oln = NULL;

    if(!urls || !urlt || !wsf || !gf)
	return NULL;

    e = find_edge_by_name(gf,urls,urlt);
    if(e)
    {
	*tl = PWEBEDGE;
	return (void *) e;
    }
    /*oln = find_independent_link_in_list(wsf->independent_links,urls,urlt);*/
    oln = find_independent_link_in_table(wsf->tindependent_links,urls,urlt);
    if(oln)
    {
	*tl = PWEBINDEPENDENTLINK;
	return (void *) oln;
    }
    else
    {
	/* *tl = PWEBINDEPENDENTLINK;
	 create new independent link  
	oln = new_web_independent_link();
	printf("New INDEPENDENT LINK %s - %s\n", urls,urlt);
	LOGR_set_link_from_name(oln, urls, urlt,wsf,gf);
	Insert_linked_list(wsf->independent_links,
			   create_new_node_llist((char *)oln));
			   return (void *)oln;*/
    }
    return NULL;
}

void
LOGR_inc_entry_page(WebGenPagePtr gpn,TypeReferer tr)
{
    switch(tr)
    {
    case RNOREFERER:
	LOGR_inc_gen_naccesses(gpn);
	break;
    case RSTATIC_REF:
	LOGR_inc_gen_nacc_static(gpn);
	break;
    case RDYNAMIC_REF:
	LOGR_inc_gen_nacc_dynamic(gpn);
	break;
    case RBOTH_REF:
	LOGR_inc_gen_nacc_static(gpn);
	LOGR_inc_gen_nacc_dynamic(gpn);
	break;
    }
}

void
LOGR_add_entry_page(WebStatPtr wsf, char *url,GraphFrame *gf,TypeReferer tr)
{
    WebGenPagePtr gpn = NULL;

    /*WebGenPagePtr gpn = find_gen_page_in_list(wsf->entry_pages,url);*/
    if(wsf)
	gpn = (WebGenPagePtr)get_element_hashed(wsf->tentry_pages, url);

    if(!gpn)
    {
	TypePage tp = 0;
	void *iptr = LOGR_find_info_gen_page(url,&tp,wsf,gf);
	/* create new entry page node  */
	gpn = new_web_gen_page_node();
	/*printf("New entry_page %s type %d\n", url,tp); */
	if(iptr)
	{
	  LOGR_set_type_gen_page(gpn,tp);
	  LOGR_set_info_gen_page(gpn,iptr);
	}
	Insert_linked_list(wsf->entry_pages,
			   create_new_node_llist((char *)gpn));
	insert_hash_info(wsf->tentry_pages, url, (char *)gpn);
    }
    /* increase count on site */
    LOGR_inc_entry_page(gpn, tr);
}

LNode_type *
LOGR_get_list_entry_pages(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->entry_pages;
}

LNode_type *
LOGR_get_alist_codes(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->alcodes;
}

WebGenInfoPtr
find_gen_code_in_list(LNode_type *lcodes,int code)
{
  LNode_type *ptr = NULL;

  if(!lcodes) return NULL;

  for(ptr = lcodes->back; ptr != lcodes; ptr = ptr->back)
  {
      WebGenInfoPtr cgpn =  (WebGenInfoPtr)ptr->info;
      int ccode = (int)LOGR_get_gen_info_context(cgpn);
      if(ccode == code)
	  return cgpn;
  }
  return NULL;
}

void
LOGR_add_http_code(WebStatPtr wsf,int code,int cur_size, int ishtmlpage)
{
  WebGenInfoPtr wgn = NULL;

  if(!wsf)
      return;

  wgn = find_gen_code_in_list(wsf->alcodes,code);

  if(!wgn)
  {
    wgn = new_gen_info_node();
    LOGR_set_gen_info_context(wgn,(void *)code);
    Insert_linked_list(wsf->alcodes,create_new_node_llist((char *)wgn));
  }

  LOGR_inc_gen_info_naccesses(wgn);
  LOGR_add_gen_info_sizebytes(wgn,cur_size);
  if(ishtmlpage)
      LOGR_inc_gen_info_npages(wgn);
}

LNode_type *
LOGR_get_alist_methods(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->almethods;
}

WebGenInfoPtr
find_gen_method_in_list(LNode_type *lmethods, HttpMethodType method)
{
  LNode_type *ptr = NULL;

  if(!lmethods) return NULL;

  for(ptr = lmethods->back; ptr != lmethods; ptr = ptr->back)
  {
      WebGenInfoPtr cgpn =  (WebGenInfoPtr)ptr->info;
     HttpMethodType cmethod = (HttpMethodType)LOGR_get_gen_info_context(cgpn);
      if(cmethod == method)
	  return cgpn;
  }
  return NULL;
}

void
LOGR_add_http_method(WebStatPtr wsf,HttpMethodType method,int cur_size, 
		     int ishtmlpage)
{
  WebGenInfoPtr wgn = NULL;

  if(!wsf)
      return;

  wgn = find_gen_method_in_list(wsf->almethods,method);

  if(!wgn)
  {
    wgn = new_gen_info_node();
    LOGR_set_gen_info_context(wgn,(void *)method);
    Insert_linked_list(wsf->almethods,create_new_node_llist((char *)wgn));
  }

  LOGR_inc_gen_info_naccesses(wgn);
  LOGR_add_gen_info_sizebytes(wgn,cur_size);
  if(ishtmlpage)
      LOGR_inc_gen_info_npages(wgn);
}


LNode_type *
LOGR_get_alist_versions(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->alversions;
}

WebGenInfoPtr
find_gen_version_in_list(LNode_type *lversions, HttpVersionType version)
{
  LNode_type *ptr = NULL;

  if(!lversions) return NULL;

  for(ptr = lversions->back; ptr != lversions; ptr = ptr->back)
  {
      WebGenInfoPtr cgpn =  (WebGenInfoPtr)ptr->info;
     HttpVersionType cversion = (HttpVersionType)LOGR_get_gen_info_context(cgpn);
      if(cversion == version)
	  return cgpn;
  }
  return NULL;
}

void
LOGR_add_http_version(WebStatPtr wsf,HttpVersionType version,int cur_size, 
		     int ishtmlpage)
{
  WebGenInfoPtr wgn = NULL;

  if(!wsf)
      return;

  wgn = find_gen_version_in_list(wsf->alversions,version);

  if(!wgn)
  {
    wgn = new_gen_info_node();
    LOGR_set_gen_info_context(wgn,(void *)version);
    Insert_linked_list(wsf->alversions,create_new_node_llist((char *)wgn));
  }

  LOGR_inc_gen_info_naccesses(wgn);
  LOGR_add_gen_info_sizebytes(wgn,cur_size);
  if(ishtmlpage)
      LOGR_inc_gen_info_npages(wgn);
}


double
LOGR_get_total_site_size(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;
  double total = 0;
  if(!wsf) return 0;

  for(ptr = wsf->lsites->back; ptr != wsf->lsites; ptr = ptr->back)
  {
      WebSiteNodePtr cwsn =  (WebSiteNodePtr)ptr->info;
      total+=LOGR_get_site_sizebytes(cwsn);
  }
  return total;
}

long
LOGR_get_total_site_naccesses(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;
  long total = 0;
  if(!wsf) return 0;

  for(ptr = wsf->lsites->back; ptr != wsf->lsites; ptr = ptr->back)
  {
      WebSiteNodePtr cwsn =  (WebSiteNodePtr)ptr->info;
      total+=LOGR_get_site_naccesses(cwsn);
  }
  return total;
}

void
LOGR_find_attribute_add_keyword(WebStatPtr wsf,HTAssocList *al,char *name,
				int se)
{
    char *value = NULL;
    char *s = "+ \"(){}";
    char *k = NULL;

    if(!wsf || !al || !name)
	return;

    value = HTAssocList_findObject(al,name);
    /*printf("value %s\n", value ? value : "NULL");*/

    if(!value)
	return;

    k = strtok(value, s);
    while (k) 
    {
      WebKeywordPtr kn = NULL;
      LOGR_string_tolower(k);
      kn = (WebKeywordPtr)get_element_hashed(wsf->keywords,k);
      if(!kn)
      {
	kn = new_web_keyword_node2(k);
	insert_hash_info(wsf->keywords, k, (char *)kn);
	Insert_linked_list(wsf->lkeywords,create_new_node_llist((char *)kn));
      }	
      LOGR_inc_total_count(kn);
      LOGR_inc_search_engine_count(kn,se);
      /*printf("KEYWORD %s Count %d Search Engine %d\n", k ? k : "NULL", 
	LOGR_get_total_count(kn),se);*/
      k = strtok(NULL, s);
    }
}

void
LOGR_add_keywords(WebStatPtr wsf, char *url, HTAssocList *al)
{
    SearchEngineType se = 0;
    
    if(!url || !wsf || !al)
	return;

    wsf->nsearch++;

    se = LOGR_get_search_engine_type(url);
    
    switch(se)
    {
    case TALTAVISTA:
	LOGR_find_attribute_add_keyword(wsf,al,"q",TALTAVISTA);
	break;
    case TYAHOO:
	LOGR_find_attribute_add_keyword(wsf,al,"p",TYAHOO);
	break;
    case TYAHOO2:
	LOGR_find_attribute_add_keyword(wsf,al,"p",TYAHOO2);
	break;
    case TGOOGLE:
	LOGR_find_attribute_add_keyword(wsf,al,"q",TGOOGLE);
	break;
    case TGOOGLE2:
	LOGR_find_attribute_add_keyword(wsf,al,"query",TGOOGLE2);
	break;
    case TINFOSEEK:
	LOGR_find_attribute_add_keyword(wsf,al,"oq",TINFOSEEK);
	LOGR_find_attribute_add_keyword(wsf,al,"ti",TINFOSEEK);
	break;
    case TLYCOS:
	LOGR_find_attribute_add_keyword(wsf,al,"query",TLYCOS);
	break;
    case TLYCOS2:
	LOGR_find_attribute_add_keyword(wsf,al,"query",TLYCOS2);
	break;
    case THOTBOT:
	LOGR_find_attribute_add_keyword(wsf,al,"MT",TLYCOS2);
	break;
    case THOTBOT2:
	LOGR_find_attribute_add_keyword(wsf,al,"query",TLYCOS2);
	break;
    }
}

Hashtable_type *
LOGR_get_table_keywords(WebStatPtr wsf)
{
    Hashtable_type *h = NULL;
    if(!wsf)
	return NULL;
    h = &(wsf->keywords);
    return h;
}

long
LOGR_get_total_number_searches(WebStatPtr wsf)
{
    if(!wsf)
	return -1;
    return wsf->nsearch;
}

void
LOGR_add_date(WebStatPtr wsf, char *sdate, int ishtmlpage, int sbytes)
{
    time_t t;
    struct tm *tm;

    int hour;
    int month;
    int day;

    WebGenInfoPtr wgi = NULL;
    WebGenInfoPtr wgim = NULL;
    WebGenInfoPtr wgid = NULL;

    if(!wsf || !sdate)
	return;

    t = HTParseTimeApache(sdate);
    tm = localtime(&t);

    hour = tm->tm_hour;
    day = tm->tm_mday-1;
    month = tm->tm_mon;

    /*printf("Date %d %d %d\n", month,day,hour);*/

    if(!wsf->hours[hour])
	wsf->hours[hour] = (void *)new_gen_info_node();

    if(!wsf->months[month])
	wsf->months[month] = (void *)new_gen_info_node();

    if(!wsf->days[day])
	wsf->days[day] = (void *)new_gen_info_node();

    wgi = (WebGenInfoPtr)wsf->hours[hour];
    wgim = (WebGenInfoPtr)wsf->months[month];
    wgid = (WebGenInfoPtr)wsf->days[day];
    
    if(ishtmlpage)
	LOGR_inc_gen_info_npages(wgi);

    LOGR_inc_gen_info_naccesses(wgi);
    LOGR_add_gen_info_sizebytes(wgi,(double)sbytes);

    if(ishtmlpage)
	LOGR_inc_gen_info_npages(wgim);

    LOGR_inc_gen_info_naccesses(wgim);
    LOGR_add_gen_info_sizebytes(wgim,(double)sbytes);

    if(ishtmlpage)
	LOGR_inc_gen_info_npages(wgid);

    LOGR_inc_gen_info_naccesses(wgid);
    LOGR_add_gen_info_sizebytes(wgid,(double)sbytes);
}


void *
LOGR_get_info_hours(WebStatPtr wsf,int i)
{
    if(!wsf)
	return NULL;
    if(i<0 && i>23)
	return NULL;
    return wsf->hours[i];
}

void *
LOGR_get_info_days(WebStatPtr wsf,int i)
{
    if(!wsf)
	return NULL;
    if(i<0 && i>30)
	return NULL;
    return wsf->days[i];
}

void *
LOGR_get_info_months(WebStatPtr wsf,int i)
{
    if(!wsf)
	return NULL;
    if(i<0 && i>11)
	return NULL;
    return wsf->months[i];
}

WebIndependentLinkPtr
find_independent_link_in_table(Hashtable_type tindependent_links, char *urls, char *urlt)
{
    char *sid = NULL;
    WebIndependentLinkPtr ol = NULL;

    if(!urls || !urlt)
	return NULL;

    sid = LOGR_gen_string_id(urls,urlt);
    if(sid)
	ol = (WebIndependentLinkPtr)get_element_hashed(tindependent_links,sid);
    if(sid) free(sid);
    return ol;
}

WebIndependentLinkPtr
find_independent_link_in_list(LNode_type *lindependent_links, char *urls, char *urlt)
{
  LNode_type *ptr = NULL;

  if(!urls || !urlt || !lindependent_links) return NULL;

  for(ptr = lindependent_links->back; ptr != lindependent_links; ptr = ptr->back)
  {
      WebIndependentLinkPtr col =  (WebIndependentLinkPtr)ptr->info;
      if(LOGR_is_independent_link(col,urls,urlt))
      {
	  return col;
      }
  }
  return NULL;
}

void
LOGR_add_independent_link(WebStatPtr wsf, char *urls,char *urlt,GraphFrame *gf,
		     char *ua)
{
    WebIndependentLinkPtr ol = NULL;

    if(!wsf || !urls || !urlt || !gf)
	return;

    if(ua && LOGR_is_user_agent_spider(wsf,ua))
    {
	/*printf("THIS IS SPIDER %s, link %s-%s\n", ua, urls,urlt);*/
	return;
    }

    if(find_edge_by_name(gf,urls,urlt))
	return;
	
    /*ol= find_independent_link_in_list(wsf->independent_links,urls,urlt);*/
    ol= find_independent_link_in_table(wsf->tindependent_links,urls,urlt);
    if(!ol)
    {
	char *sid = LOGR_gen_string_id(urls,urlt);
	/* create new independent link  */
	ol = new_web_independent_link();
	/*printf("New independent_link %s - %s\n", urls,urlt);*/
	LOGR_set_link_from_name(ol, urls, urlt,wsf,gf);
	Insert_linked_list(wsf->independent_links,
			   create_new_node_llist((char *)ol));
	insert_hash_info(wsf->tindependent_links, sid, (char *)ol);
	if(sid) free(sid);
    }
    else
    {
	/*int nacc = LOGR_get_link_naccesses(ol);
	if(nacc == 0)
	{
	  printf("Updating independent LINK %s - %s\n", urls, urlt);
	  LOGR_set_link_sizebytes(ol,size);
	  }*/
    }
    /* increase count on independent link */
    LOGR_inc_link_naccesses(ol);
}

LNode_type *
LOGR_get_list_independent_links(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->independent_links;
}

short
LOGR_is_user_agent_spider(WebStatPtr wsf,char *ua_name)
{
    WebBrowserNodePtr wbn = NULL;
    short iua = 0;
    if(!wsf || !ua_name)
	return 0;
    
    /*wbn = find_browser_in_list(wsf->luas, ua_name);*/
    wbn = (WebBrowserNodePtr)get_element_hashed(wsf->tuas,ua_name);
    
    if(wbn) {
	iua = (short)LOGR_is_spider(wbn);
	return iua;
    }
    return 0;
}

LNode_type *
LOGR_get_list_directories(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->ldirectories;
}

LNode_type *
LOGR_get_list_graphs(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lgraphs;
}

LNode_type *
LOGR_get_list_paths(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lpaths;
}

LNode_type *
LOGR_get_list_users(WebStatPtr wsf)
{
  if(!wsf) return NULL;
  return wsf->lusers;
}

WebUgraphPtr
LOGR_find_ugraph_in_graphs_list(LNode_type *lgraphs, WebUgraphPtr ugn)
{
  LNode_type *ptr = NULL;
  void * GArray = LOGR_get_ugraph_info(ugn);

  if(!lgraphs || !ugn) return NULL;

  for(ptr = lgraphs->back; ptr != lgraphs; ptr = ptr->back)
  {
      WebUgraphPtr cwun =  (WebUgraphPtr)ptr->info;
      void * HArray = LOGR_get_ugraph_info(cwun);

      if(LOGR_is_graphs_equal(GArray,HArray))
	  return cwun;
  }
  return NULL;
}

WebPathPtr
LOGR_find_path_in_list(LNode_type *lpaths, WebPathPtr wpn)
{
  LNode_type *ptr = NULL;

  if(!wpn || !lpaths) return NULL;

  for(ptr = lpaths->back; ptr != lpaths; ptr = ptr->back)
  {
      WebPathPtr cwpn =  (WebPathPtr)ptr->info;
      if(LOGR_is_paths_equal(cwpn,wpn))
	  return cwpn;
  }
  return NULL;
}

WebGenPagePtr
LOGR_find_directory_in_list(LNode_type *ldirectories, char *dirname)
{
  LNode_type *ptr = NULL;

  if(!dirname || !ldirectories) return NULL;

  for(ptr = ldirectories->back; ptr != ldirectories; ptr = ptr->back)
  {
      WebGenPagePtr dn =  (WebGenPagePtr)ptr->info;
      char *dname = (char *)LOGR_get_info_gen_page(dn);
      if(dname && !strcmp(dirname,dname))
	  return dn;
  }
  return NULL;
}

void
LOGR_add_current_directory(WebStatPtr wsf, char *url, int hits,int bytes,
			   int i)
{
    WebGenPagePtr dn = NULL;
    if(!wsf || !url)
	return;
    /*dn = LOGR_find_directory_in_list(wsf->ldirectories,url);*/
    if(wsf)
	dn = (WebGenPagePtr)get_element_hashed(wsf->tdirectories,url);
    if(!dn)
    {
	char *nurl = NULL;
	dn = new_web_gen_page_node();
	GRStrAllocCopy(nurl,url);
	LOGR_set_info_gen_page(dn,(void *)nurl);
	Insert_linked_list(wsf->ldirectories,
			   create_new_node_llist((char *)dn));
	insert_hash_info(wsf->tdirectories, url, (char *)dn);
    }
    if(!i)
    {
	LOGR_add_gen_total_bytes(dn,(long)hits*(long)bytes);
	LOGR_add_gen_nacc_static(dn,(long)hits);
    }
    LOGR_add_gen_naccesses(dn,(long)hits);
}

void
LOGR_add_directory(WebStatPtr wsf, char *url, int hits,int bytes)
{
    char *p = NULL;
    char *t = NULL;
    int i = 0;

    if(!wsf || !url)
	return;

    if(strchr(url,'?'))
	return;

    p = strrchr(url,'/');
    while(p && (int)(p-url) > 8)
    {
	*p = '\0';
	LOGR_add_current_directory(wsf,url,hits,bytes,i);
	t = p;
	p = strrchr(url,'/');
	*t = '/';
	i++;
    }
}

void
LOGR_add_user_entry_page_by_name(WebUserPtr usn,WebStatPtr wsf, char *url)
{
    WebGenPagePtr gpp = NULL;

    LNode_type *lepages = NULL;
    char *entry_page = NULL;

    if(!usn || !wsf || !url)
	return;
    
    entry_page = LOGR_get_user_entry_page(usn);

    if(!entry_page)
	return;

    if(!strcmp(entry_page,url))
	return;
	
    lepages = LOGR_get_user_lepages(usn);

    gpp = find_gen_page_in_list(lepages,url);

    if(gpp)
	return;

    /*gpp = find_gen_page_in_list(wsf->entry_pages,url);*/
    gpp = (WebGenPagePtr)get_element_hashed(wsf->tentry_pages, url);

    if(gpp)
	LOGR_add_user_entry_page(usn,(void*)gpp);
    else
	fprintf(stderr,"ENTRY PAGE no found %s\n", url);
}

void
LOGR_get_total_number_pages(WebStatPtr wsf,GraphFrame *gf,int *nhtml,
			    int *nnhtml, int *nobj)
{
  int tnhtml = 0;
  int tnnhtml = 0;
  int tnobj = 0;
  
  LNode_type *nodes = get_list_vertices(gf); 
  LNode_type *ptr = NULL;
  LNode_type *lop = LOGR_get_list_independent_pages(wsf);

  if(!gf || !wsf)
      return;

  for(ptr = nodes->back; ptr != nodes; ptr=ptr->back)
  {
    PNODE v = (PNODE)ptr->info;
    const char *smime = get_attribute(v, "mime");

    if(get_counter(v)== 0)
	continue;

    if(smime)
    {
      if(is_vertex_object(v))
	tnobj++;
      else if(!strcmp(smime,"text/html"))
	tnhtml++;
      else
	tnnhtml++;
    }
  }

  for(ptr = lop->back; ptr != lop; ptr = ptr->back)
  {
    WebIndependentPagePtr copn =  (WebIndependentPagePtr)ptr->info;
    const char *smime = LOGR_get_ip_mime(copn);
    long size = LOGR_get_ip_sizebytes(copn);
    if(size > 0)
    {
      if(!strcmp(smime,"text/html"))
	  tnhtml++;
      else if(!strcmp(smime,"www/unknown"))
	  tnhtml++;
      else
	  tnnhtml++;
    }
  }

  *nhtml = tnhtml;
  *nnhtml = tnnhtml;
  *nobj = tnobj;
}

void
LOGR_get_total_number_hyperlinks(WebStatPtr wsf,GraphFrame *gf,int *hhtml,
				 int *hnhtml)
{
  int th = 0;
  int tnh = 0;

  LNode_type *nodes = get_list_vertices(gf); 
  LNode_type *lol = LOGR_get_list_independent_links(wsf);
  LNode_type *ptr = NULL;

  if(!wsf || !gf)
      return;

  for(ptr = nodes->back; ptr != nodes; ptr=ptr->back)
    {
      PNODE v = (PNODE)ptr->info;
      LNode_type *peo, *ledges_out = get_edges_out(v);
      for(peo = ledges_out->back ; peo != ledges_out ; peo = peo->back)
	{
	  PEDGE e = (PEDGE)peo->info;
	  PNODE u = get_to_vertex(e);
	  char *we = get_weight_edge(e);
	  int cedge = we ? atoi(we) : 0;
	  const char *smime = get_attribute(u,"mime");

	  if(get_edge_attribute(e) == 'z')
	    continue;

	  if(cedge > 0 && smime)
	  {
	      if(!strcmp(smime,"text/html"))
		  th++;	
	      else
		  tnh++;
		 
	  }
	}
    }

  for(ptr = lol->back; ptr != lol; ptr = ptr->back)
  {
      WebIndependentLinkPtr col =  (WebIndependentLinkPtr)ptr->info;
      const char *smime = LOGR_get_link_target_mime(col);	
      if(!smime)
	  continue;
      if(!strcmp(smime,"text/html"))
	  th++;	
      else if(!strcmp(smime,"www/unknown"))
	  th++;	
      else
	  tnh++;
  }

  *hhtml = th;
  *hnhtml = tnh;
}

void
LOGR_get_total_number_entry_pages(WebStatPtr wsf,GraphFrame *gf,int *ehtml,
				 int *enhtml)
{
  int tnhtml = 0;
  int tnnhtml = 0;
  LNode_type *lentrys = LOGR_get_list_entry_pages(wsf);
  LNode_type *ptr = NULL;

  if(!wsf || !gf)
      return;

  for(ptr = lentrys->back; ptr != lentrys; ptr=ptr->back)
  {
    WebGenPagePtr mop = (WebGenPagePtr)ptr->info;
    const char *mime = LOGR_get_gen_mime(mop);
    if(mime)
    {
	if(!strcmp(mime,"text/html") || !strcmp(mime,"www/unknown"))
	    tnhtml++;
	else
	    tnnhtml++;
    }
  }
  *ehtml = tnhtml;
  *enhtml = tnnhtml;
}

void
LOGR_get_total_number_referers(WebStatPtr wsf,GraphFrame *gf,int *rhost,
			       int *rsa)
{
  LNode_type *lsites_ref = LOGR_get_list_sites_ref(wsf);
  LNode_type *ptr = NULL;

  int tnrefhost = 0;
  int tnrefsengine = 0;
  
  if(!wsf || !gf)
      return;

  for(ptr = lsites_ref->back; ptr != lsites_ref; ptr=ptr->back)
  {
    WebSiteNodePtr msite = (WebSiteNodePtr)ptr->info;
    TypeReferer tr = LOGR_get_site_type_referer(msite);
    if(tr & RDYNAMIC_REF) 
	tnrefsengine++;
    if (tr == RNOREFERER)
	tnrefhost++;
    if (tr & RSTATIC_REF)
	tnrefhost++;
  }

  *rhost = tnrefhost;
  *rsa = tnrefsengine;
}


void
LOGR_get_average_by_time(WebStatPtr wsf,double *ave_req_day,
			 double *ave_req_hour, double *ave_size_day, 
			 double *ave_size_hour)
{
  long tbytes = LOGR_get_total_site_size(wsf);
  long tacc = LOGR_get_total_site_naccesses(wsf);

  time_t st;
  time_t et;
  double dt = 0;
  double nhours = 0;
  double ndays = 0;
  

  LNode_type *lsd = LOGR_get_start_dates(wsf);
  LNode_type *sptr = lsd->next;
  LNode_type *led = LOGR_get_end_dates(wsf);
  LNode_type *eptr = led->next;

  for(;sptr!=lsd; sptr=sptr->next,eptr=eptr->next)
  {
    st = (time_t)sptr->info;
    et = (time_t)eptr->info;

    dt += (double)fabs(difftime(st,et));
  }

  nhours = dt / 3600.0;
  ndays = nhours / 24.0;

  *ave_req_hour = (double)tacc/nhours;
  *ave_req_day = (double)tacc/ndays;
  *ave_size_hour = (double)tbytes/nhours;
  *ave_size_day = (double)tbytes/ndays;
}

long
LOGR_get_entry_page_hits(WebStatPtr wsf,char *url)
{

    WebGenPagePtr gpn = find_gen_page_in_list(wsf->entry_pages,url);
    return gpn ? LOGR_get_gen_naccesses(gpn) : 0;
}

void
clear_directories(WebStatPtr msn)
{
    LNode_type *ptr = NULL;
    LNode_type *ldir = msn->ldirectories;
    WebGenPagePtr dn = NULL;
    for(ptr = ldir->back; ptr !=ldir ; ptr=ptr->back)
    {
	dn = (WebGenPagePtr)ptr->info;
	LOGR_set_gen_total_bytes(dn,0);
	LOGR_set_gen_nacc_static(dn,0);
	LOGR_set_gen_naccesses(dn,0);
    }
}

void
process_directories_of_nodes_webgraph(GraphFrame *gf,WebStatPtr msn)
{
  LNode_type *nodes = get_list_vertices(gf);
  LNode_type *ptr = nodes->back;
  clear_directories(msn);
  for(; ptr != nodes; ptr=ptr->back)
  {
    PNODE v = (PNODE)ptr->info;
    int hits = get_counter(v);
    char *vlabel = get_label_vertex(v);
    char *ssize = (char *)get_attribute(v,"size");
    int size = ssize ? atoi(ssize) : 0;
    if(hits > 0)
	LOGR_add_directory(msn,vlabel,hits,size);
  }
}

void 
process_directories_of_nodes_independent(WebStatPtr msn)
{
    LNode_type *lips = LOGR_get_list_independent_pages(msn);
    LNode_type *ptr = lips->back;
    for(ptr = lips->back; ptr != lips; ptr=ptr->back)
    {
	WebIndependentPagePtr mip = (WebIndependentPagePtr)ptr->info;
	char *label = (char *)LOGR_get_ip_url(mip);
	int hits = (int)LOGR_get_ip_naccesses(mip);
	long size = LOGR_get_ip_sizebytes(mip);
	if(hits > 0)
	    LOGR_add_directory(msn,label,hits,(int)size);
    }
    return;
}

void
LOGR_process_directories(WebStatPtr wsf,GraphFrame *gf)
{
    process_directories_of_nodes_webgraph(gf,wsf);
    process_directories_of_nodes_independent(wsf);
}

void
LOGR_process_sites(WebStatPtr wsf,int what)
{
  LNode_type *ptr = NULL;
  int timeout = 0;
  int i = 0;

  if(!wsf) 
      return;
  timeout =  LOGR_get_resolve_timeout(wsf);
  fprintf(stderr,"Start Resolving IP addresses (timeout %d)...\n",timeout);

  for(ptr = wsf->lsites->back; ptr != wsf->lsites; ptr = ptr->back)
  {
      WebSiteNodePtr wsn = (WebSiteNodePtr)ptr->info;
      const char *csite = wsn ? LOGR_get_site_dns(wsn) : NULL;
      const char *ipsite = wsn ? LOGR_get_site_ip(wsn) : NULL;
      char *lsite = NULL;

      if(!csite && ipsite && what)
      {
	  i++;
	  lsite = LOGR_hostfromIP((char *)ipsite,timeout);
	  /*printf("IP %s DNS %s\n", ipsite, lsite ? lsite : "NULL");*/
	  LOGR_assign_site_dns(wsn,lsite);
      }
      else if(!ipsite && csite && !what)
      {
	  lsite = LOGR_IPfromhost((char *)csite);
	  /*printf("DNS %s IP %s\n", csite, lsite ? lsite : "NULL");*/
	  LOGR_assign_site_ip(wsn,lsite);
      }
  }
  fprintf(stderr,"\n... Finish Resolving %d IP addresses\n",i);
}

void
clear_domains(WebStatPtr wsf)
{
    LNode_type *ptr = NULL;
    LNode_type *ldomains = wsf->ldomains;
    WebGenInfoPtr in = NULL;
    
    for(ptr = ldomains->back; ptr != ldomains; ptr=ptr->back)
    {
	in = (WebGenInfoPtr)ptr->info;
	LOGR_set_gen_info_sizebytes(in,0);
	LOGR_set_gen_info_naccesses(in,0);
	LOGR_set_gen_info_npages(in,0);
    }
}


void
LOGR_process_domains(WebStatPtr wsf)
{
  LNode_type *ptr = NULL;

  if(!wsf) 
      return;

  clear_domains(wsf);

  for(ptr = wsf->lsites->back; ptr != wsf->lsites; ptr = ptr->back)
  {
      WebSiteNodePtr wsn = (WebSiteNodePtr)ptr->info;
      const char *csite = wsn ? LOGR_get_site_dns(wsn) : NULL;
      long nacc = wsn ? LOGR_get_site_naccesses(wsn) : 0;
      long size = wsn ? LOGR_get_site_sizebytes(wsn) : 0;
      long npages = wsn ? LOGR_get_site_nacc_search(wsn) : 0;
      
      if(csite)
      {
	  char *p = strrchr(csite,'.');
	  if(p)
	  {
	      p++;
	      LOGR_add_domain(wsf,p,nacc,size,npages);
	      continue;
	  }
      }
      LOGR_add_domain(wsf,"Unknown",nacc,size,npages);
  }
}

WebSimplePagePtr 
find_page_info_simple_list(LNode_type *simple_pages,void *info)
{
    void *pinfo = NULL;
    LNode_type *ptr = NULL;
    WebSimplePagePtr spn = NULL;

    if(!simple_pages || !info)
	return NULL;

    for(ptr = simple_pages->back; ptr != simple_pages; ptr = ptr->back)
    {
	spn = (WebSimplePagePtr)(ptr->info);
	pinfo = LOGR_get_info_simple_page(spn);
	
	if(pinfo == info)
	    return spn;
    }
    return NULL;
}

void
LOGR_add_simple_page(WebStatPtr wsf, TypePage tp, void *info, short verify)
{
    WebSimplePagePtr spn = NULL;

    if(!info)
	return;

    if(verify)
    {
	spn = find_page_info_simple_list(wsf->simple_pages,info);
	if(spn)
	    return;
    }

    /* create new simple page node  */
    spn = new_web_simple_page_node();
    LOGR_set_type_simple_page(spn,tp);
    LOGR_set_info_simple_page(spn,info);

    /*printf("New simple_page %s\n", LOGR_get_simple_url(spn));*/

    Insert_linked_list(wsf->simple_pages,create_new_node_llist((char *)spn));
}

void
LOGR_process_simple_pages(WebStatPtr wsf,GraphFrame *gf,short verify)
{
    LNode_type *lips = LOGR_get_list_independent_pages(wsf);
    LNode_type *nodes = get_list_vertices(gf);
    LNode_type *ptr = NULL;

    for(ptr = lips->back; ptr != lips; ptr=ptr->back)
    {
	WebIndependentPagePtr mip = (WebIndependentPagePtr)ptr->info;
	int hits = LOGR_get_ip_naccesses(mip);
	if(hits > 0)
	    LOGR_add_simple_page(wsf,PWEBINDEPENDENT,(void*)mip,verify);
    }

    for(ptr = nodes->back; ptr != nodes; ptr=ptr->back)
    {
	PNODE v = (PNODE)ptr->info;
	int hits = get_counter(v);
	if(hits > 0)
	    LOGR_add_simple_page(wsf,PWEBNODE,(void*)v,verify);
    }
}

void
verify_gen_link_in_list(LNode_type *slinks, WebGenLinkPtr spn)
{
    LNode_type *ptr = NULL;
    for(ptr = slinks->back; ptr != slinks; ptr = ptr->back)
    {
	WebGenLinkPtr cpn = (WebGenLinkPtr)ptr->info;
	if(LOGR_is_gen_links_equal(cpn,spn))
	    fprintf(stderr,"Same link %s - %s\n", 
		    LOGR_get_gen_url_source(spn),
		    LOGR_get_gen_url_target(spn));

    }
}

WebGenLinkPtr
find_link_info_simple_list(LNode_type *simple_links,void *info)
{
    void *pinfo = NULL;
    LNode_type *ptr = NULL;
    WebGenLinkPtr spn = NULL;

    if(!simple_links || !info)
	return NULL;

    for(ptr = simple_links->back; ptr != simple_links; ptr = ptr->back)
    {
	spn = (WebGenLinkPtr)(ptr->info);
	pinfo = LOGR_get_info_gen_link(spn);
	
	if(pinfo == info)
	    return spn;
    }
    return NULL;
}

void
LOGR_add_simple_link(WebStatPtr wsf, TypeLink tl, void *info, short verify)
{
    WebGenLinkPtr spn = NULL;

    if(!info)
	return;

    if(verify)
    {
	spn = find_link_info_simple_list(wsf->simple_links,info);
	if(spn)
	    return;
    }

    /* create new simple link node  */
    spn = new_web_gen_link();
    LOGR_set_type_gen_link(spn,tl);
    LOGR_set_info_gen_link(spn,info);

    /*verify_gen_link_in_list(wsf->simple_links,spn);*/

    /*printf("New simple link %d - %d\n", LOGR_get_gen_link_source_num(spn),
      LOGR_get_gen_link_target_num(spn));*/

    Insert_linked_list(wsf->simple_links,create_new_node_llist((char *)spn));
}

void
LOGR_process_simple_links(WebStatPtr wsf,GraphFrame *gf, short verify)
{  
    LNode_type *independent_links = LOGR_get_list_independent_links(wsf);
    LNode_type *nodes = get_list_vertices(gf);
    LNode_type *ptr = NULL;

    for(ptr = independent_links->back; ptr != independent_links; 
	ptr=ptr->back)
    {
	WebIndependentLinkPtr il = (WebIndependentLinkPtr)ptr->info;
	int hits = il ? LOGR_get_link_naccesses(il) : 0;
	if(hits > 0)
	    LOGR_add_simple_link(wsf,PWEBINDEPENDENTLINK,(void*)il,verify);
    }
    
    for(ptr = nodes->back; ptr != nodes; ptr=ptr->back)
    {
	PNODE v = (PNODE)ptr->info;
	LNode_type *peo, *ledges_out = get_edges_out(v);
	for(peo = ledges_out->back ; peo != ledges_out ; peo = peo->back)
    	{
	    PEDGE e = (PEDGE)peo->info;
	    char *we = get_weight_edge(e);
	    int cedge = we ? atoi(we) : 0;
	    if(cedge > 0)
	    {
		LOGR_add_simple_link(wsf,PWEBEDGE,(void*)e,verify);
	    }
	}
    }
}





